Advanced Lane Finding Project¶
The goals / steps of this project are the following:
- Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
- Apply a distortion correction to raw images.
- Use color transforms, gradients, etc., to create a thresholded binary image.
- Apply a perspective transform to rectify binary image ("birds-eye view").
- Detect lane pixels and fit to find the lane boundary.
- Determine the curvature of the lane and vehicle position with respect to center.
- Warp the detected lane boundaries back onto the original image.
- Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.
First, I'll compute the camera calibration using chessboard images¶
In [1]:
# Video Pipeline for Project 4 - Car ND - Advanced Lane detection
# Advanced Lane Finding Project
# The goals / steps of this project are the following:
# Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
# Apply a distortion correction to raw images.
# Use color transforms, gradients, etc., to create a thresholded binary image.
# Apply a perspective transform to rectify binary image ("birds-eye view").
# Detect lane pixels and fit to find the lane boundary.
# Determine the curvature of the lane and vehicle position with respect to center.
# Warp the detected lane boundaries back onto the original image.
# Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import glob
import pdb
%matplotlib inline
test_mode = False
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
# Chessboard Dimensions
nx = 9
ny = 6
In [2]:
# Method for printing images
def print_image(img, title = ""):
if test_mode == True:
plt.figure(figsize=(7, 7))
plt.imshow(img)
plt.title(title)
def print_gray_image(img, title = ""):
if test_mode == True:
plt.figure(figsize=(7, 7))
plt.imshow(img, cmap="gray")
plt.title(title)
In [3]:
calibration_path = '/Users/Akshay/projects/carnd/CarND-Advanced-Lane-Lines/camera_cal/calibration*.jpg'
test_image = '/Users/Akshay/projects/carnd/CarND-Advanced-Lane-Lines/camera_cal/calibration1.jpg'
test_image = mpimg.imread(test_image)
calibration_images = glob.glob(calibration_path)
# Store Image Points and Object points for image
image_points = []
object_points = []
obj_points = np.zeros((nx * ny, 3), np.float32)
obj_points[:, :2] = np.mgrid[0:nx, 0:ny].T.reshape(-1, 2)
for img_name in calibration_images:
img = mpimg.imread(img_name)
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)
if ret == True:
image_points.append(corners)
object_points.append(obj_points)
def calibrate_camera(img, image_points, object_points):
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
ret, corners = cv2.findChessboardCorners(gray, gray.shape, None)
img = cv2.drawChessboardCorners(img, gray.shape, corners, ret)
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(object_points, image_points, gray.shape[::-1], None, None)
undist = cv2.undistort(img, mtx, dist, None, mtx)
return undist, mtx, dist
In [4]:
undist_img, pers_mtx, dist_coeff = calibrate_camera(test_image, image_points, object_points)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(test_image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(undist_img)
ax2.set_title('Undistorted Image', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
In [5]:
# Useful for masking out unwanted part of image
def region_of_interest(img, vertices):
"""
Applies an image mask.
Only keeps the region of the image defined by the polygon
formed from `vertices`. The rest of the image is set to black.
"""
# defining a blank mask to start with
mask = np.zeros_like(img)
# defining a 3 channel or 1 channel color to fill the mask with depending on the input image
if len(img.shape) > 2:
channel_count = img.shape[2] # i.e. 3 or 4 depending on your image
ignore_mask_color = (255,) * channel_count
else:
ignore_mask_color = 255
#filling pixels inside the polygon defined by "vertices" with the fill color
cv2.fillPoly(mask, vertices, ignore_mask_color)
# returning the image only where mask pixels are nonzero
masked_image = cv2.bitwise_and(img, mask)
return masked_image
# Thresholding functions. Applies sobel filter to image
def abs_sobel_thresh(img, orient = 'x', sobel_kernel = 3, thresh = (0, 255)):
# Convert to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Apply x or y gradient with the OpenCV Sobel() function
# and take the absolute value
if orient == 'x':
abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0))
if orient == 'y':
abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1))
# Rescale back to 8 bit integer
scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
# Create a copy and apply the threshold
binary_output = np.zeros_like(scaled_sobel)
# Here I'm using inclusive (>=, <=) thresholds, but exclusive is ok too
binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1
# Return the result
return binary_output
# Calculates magnitude and applies thresholding
def mag_thresh(image, sobel_kernel = 3, mag_thresh = (0, 255)):
# Convert to grayscale
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
# Take both Sobel x and y gradients
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
# Calculate the gradient magnitude
gradmag = np.sqrt(sobelx**2 + sobely**2)
# Rescale to 8 bit
scale_factor = np.max(gradmag)/255
gradmag = (gradmag/scale_factor).astype(np.uint8)
# Create a binary image of ones where threshold is met, zeros otherwise
binary_output = np.zeros_like(gradmag)
binary_output[(gradmag >= mag_thresh[0]) & (gradmag <= mag_thresh[1])] = 1
# Return the binary image
return binary_output
# applies thresholds to image along the axes.
def dir_threshold(image, sobel_kernel = 3, thresh = (0, 255)):
# Grayscale
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
# Calculate the x and y gradients
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
# Take the absolute value of the gradient direction,
# apply a threshold, and create a binary image result
absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
binary_output = np.zeros_like(absgraddir)
binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1
# Return the binary image
return binary_output
# function to get the s_channel from the image
def hls_select(img, thresh=(0, 255)):
# 1) Convert to HLS color space
in_hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
# 2) Apply a threshold to the S channel
s_channel = in_hls[:,:,2]
binary_output = np.zeros_like(s_channel)
# 3) Return a binary image of threshold result
binary_output[(s_channel > thresh[0]) & (s_channel <= thresh[1])] = 1
return binary_output
# function to get the L channel from the image
def l_select(img, thresh=(0, 255)):
# 1) Convert to HLS color space
in_hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
# 2) Apply a threshold to the S channel
l_channel = in_hls[:,:,1]
binary_output = np.zeros_like(l_channel)
# 3) Return a binary image of threshold result
binary_output[(l_channel > thresh[0]) & (l_channel <= thresh[1])] = 1
return binary_output
# Applies thresholding and returns a usable thresholded image
def extract_combined_binary(current_image):
ksize = 15
# Gradient threshold along the X axis
gradx = abs_sobel_thresh(current_image, orient='x', sobel_kernel=ksize, thresh=(20, 100))
# Gradient threshold along the Y axis
grady = abs_sobel_thresh(current_image, orient='y', sobel_kernel=ksize, thresh=(20, 100))
mag_binary = mag_thresh(current_image, sobel_kernel = ksize, mag_thresh=(25, 100))
if test_mode:
print_gray_image(mag_binary, "Magnitude Binary")
dir_binary = dir_threshold(current_image, sobel_kernel = 15, thresh=(0.7, 1.3))
if test_mode:
print_gray_image(dir_binary, "Directional Binary")
combined = np.zeros_like(dir_binary)
combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1
binary_color_combined = np.zeros_like(combined)
s_binary = hls_select(current_image, thresh=(90, 200))
l_binary = l_select(current_image, thresh=(0, 110))
# Visual how images look like in S and L channels.
# I was going to make use of the L channel but later on decided
# I was getting results by tweaking other params and using only S channel
if test_mode:
print_gray_image(s_binary, "S Channel Binary")
if test_mode:
print_gray_image(l_binary, "L Channel Binary")
binary_color_combined[((combined == 1) | (s_binary == 1))] = 1
# binary_color_combined[(l_binary == 1)] = 0
binary_color_combined[(l_binary == 1)] = 0
return binary_color_combined
# Perspective transform warping. SRC and DST points are manually
# set for simplicity.
def perform_binary_warp(binary_image):
# Use input_image for perspective transform
binary_image_orig = np.copy(binary_image)
src_points_array = np.array([[550, 480], [750, 480], [1100, 700], [320, 700]])
dst_points_array = np.array([[200, 100], [1100, 100], [1100, 650], [200, 650]])
src_points = np.int32(src_points_array)
dst_points = np.int32(dst_points_array)
src_points = src_points.reshape((-1,1,2))
dst_points = dst_points.reshape((-1,1,2))
cv2.polylines(binary_image_orig, [src_points], True, (20, 20, 20), 2)
cv2.polylines(binary_image_orig, [dst_points], True, (20, 20, 20), 2)
print_image(binary_image_orig, "Perspective Transform mappings")
# start warping the image
src_points = np.float32(src_points_array)
dst_points = np.float32(dst_points_array)
M = cv2.getPerspectiveTransform(src_points, dst_points)
Minv = cv2.getPerspectiveTransform(dst_points, src_points)
img_size = (binary_image.shape[1], binary_image.shape[0])
binary_warped = cv2.warpPerspective(binary_image, M, img_size)
return binary_warped, M, Minv
In [6]:
left_lane_base = 0
right_lane_base = 0
def fit_polynomial_with_sliding_windows(binary_warped):
global left_lane_base
global right_lane_base
# Create an output image to draw on and visualize the result
visualization_image = np.dstack((binary_warped, binary_warped, binary_warped))*255
# Assuming you have created a warped binary image called "binary_warped"
# Take a histogram of the bottom half of the image
histogram = np.sum(binary_warped[binary_warped.shape[0]/2:, :], axis = 0)
plt.plot(histogram)
# Find the peak of the left and right halves of the histogram
# These will be the starting point for the left and right lines
midpoint = np.int(histogram.shape[0]/2)
leftx_base = np.argmax(histogram[:midpoint])
rightx_base = np.argmax(histogram[midpoint:]) + midpoint
# Choose the number of sliding windows
nwindows = 9
# Set height of windows
window_height = np.int(binary_warped.shape[0]/nwindows)
# Identify the x and y positions of all nonzero pixels in the image
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Current positions to be updated for each window
leftx_current = leftx_base
rightx_current = rightx_base
# Track centers of current image's lane.
left_lane_base = leftx_current
right_lane_base = rightx_current
# Helps with smoothing the lane line.
# 1000/1150 works well as a threshold for the right lane line.
if rightx_current < 1000:
rightx_current = 1150
# Set the width of the windows +/- margin
margin = 100
# Set minimum number of pixels found to recenter window
minpix = 50
# Create empty lists to receive left and right lane pixel indices
left_lane_inds = []
right_lane_inds = []
# Step through the windows one by one
for window in range(nwindows):
# Identify window boundaries in x and y (and right and left)
win_y_low = binary_warped.shape[0] - (window+1)*window_height
win_y_high = binary_warped.shape[0] - window*window_height
win_xleft_low = leftx_current - margin
win_xleft_high = leftx_current + margin
win_xright_low = rightx_current - margin
win_xright_high = rightx_current + margin
# Draw the windows on the visualization image
cv2.rectangle(visualization_image,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),(0,255,0), 2)
cv2.rectangle(visualization_image,(win_xright_low,win_y_low),(win_xright_high,win_y_high),(0,255,0), 2)
# Identify the nonzero pixels in x and y within the window
good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]
good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]
# Append these indices to the lists
left_lane_inds.append(good_left_inds)
right_lane_inds.append(good_right_inds)
# If you found > minpix pixels, recenter next window on their mean position
if len(good_left_inds) > minpix:
leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
if len(good_right_inds) > minpix:
rightx_current = np.int(np.mean(nonzerox[good_right_inds]))
# Concatenate the arrays of indices
# Flattening out the array
left_lane_inds = np.concatenate(left_lane_inds)
right_lane_inds = np.concatenate(right_lane_inds)
# Extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
# return left_fit, right_fit, leftx, lefty, rightx, righty
if test_mode:
##### Visualization starts here #####
# Generate x and y values for plotting
ploty = np.linspace(0, binary_warped.shape[0] - 1, binary_warped.shape[0])
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
visualization_image[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
visualization_image[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]
plt.imshow(visualization_image)
plt.plot(left_fitx, ploty, color='yellow')
plt.plot(right_fitx, ploty, color='yellow')
plt.xlim(0, 1280)
plt.ylim(720, 0)
##### Visualization 2 #####
# Create an image to draw on and an image to show the selection window
visualization_image = np.dstack((binary_warped, binary_warped, binary_warped))*255
window_img = np.zeros_like(visualization_image)
# Color in left and right line pixels
visualization_image[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
visualization_image[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]
# Generate a polygon to illustrate the search window area
# And recast the x and y points into usable format for cv2.fillPoly()
left_line_window1 = np.array([np.transpose(np.vstack([left_fitx-margin, ploty]))])
left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_fitx+margin, ploty])))])
left_line_pts = np.hstack((left_line_window1, left_line_window2))
right_line_window1 = np.array([np.transpose(np.vstack([right_fitx-margin, ploty]))])
right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx+margin, ploty])))])
right_line_pts = np.hstack((right_line_window1, right_line_window2))
# Draw the lane onto the warped blank image
cv2.fillPoly(window_img, np.int_([left_line_pts]), (0,255, 0))
cv2.fillPoly(window_img, np.int_([right_line_pts]), (0,255, 0))
result = cv2.addWeighted(visualization_image, 1, window_img, 0.3, 0)
plt.imshow(result)
plt.plot(left_fitx, ploty, color='yellow')
plt.plot(right_fitx, ploty, color='yellow')
plt.xlim(0, 1280)
plt.ylim(720, 0)
return left_fit, right_fit, leftx, lefty, rightx, righty
In [7]:
def poly_fit_fast(binary_warped, left_fit, right_fit):
# Assume you now have a new warped binary image
# from the next frame of video (also called "binary_warped")
# It's now much easier to find line pixels!
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
margin = 100
left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2] - margin)) &
(nonzerox < (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2] + margin)))
right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2] - margin)) &
(nonzerox < (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2] + margin)))
# Again, extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
return left_fit, right_fit, leftx, lefty, rightx, righty
In [8]:
def measure_radius_of_curvature(binary_warped, left_fit, right_fit, leftx, lefty, rightx, righty):
# linspace returns a lineear distribution of points between param1 and param2.
ploty = np.linspace(0, binary_warped.shape[0] - 1, binary_warped.shape[0])
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
margin = 100
left_line_window1 = np.array([np.transpose(np.vstack([left_fitx-margin, ploty]))])
left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_fitx+margin, ploty])))])
left_line_pts = np.hstack((left_line_window1, left_line_window2))
right_line_window1 = np.array([np.transpose(np.vstack([right_fitx-margin, ploty]))])
right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx+margin, ploty])))])
right_line_pts = np.hstack((right_line_window1, right_line_window2))
y_eval = np.max(ploty)
left_curverad = ((1 + (2*left_fit[0]*y_eval + left_fit[1])**2)**1.5) / np.absolute(2*left_fit[0])
right_curverad = ((1 + (2*right_fit[0]*y_eval + right_fit[1])**2)**1.5) / np.absolute(2*right_fit[0])
# Define conversions in x and y from pixels space to meters
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = 3.7/700 # meters per pixel in x dimension
# Fit new polynomials to x, y in world space
left_fit_cr = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
right_fit_cr = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)
# Calculate the new radii of curvature in real world space
left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
# Calculate offset of vehicle from center of the lane
image_center = current_image.shape[1] / 2
lane_center = (right_lane_base - left_lane_base) / 2
offset_from_center = image_center - lane_center
offset_from_center *= xm_per_pix
return left_curverad, right_curverad, ploty, offset_from_center
In [9]:
# Projects the detected lane on image
def project_detection_on_lane(warped, image, left_fit, right_fit, ploty, Minv):
ploty = np.linspace(0, warped.shape[0] - 1, warped.shape[0])
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
# # Create an image to draw the lines on
warp_zero = np.zeros_like(warped).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
# Recast the x and y points into usable format for cv2.fillPoly()
pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
pts = np.hstack((pts_left, pts_right))
# Draw the lane onto the warped blank image
cv2.fillPoly(color_warp, np.int_([pts]), (0, 255, 0))
# Warp the blank back to original image space using inverse perspective matrix (Minv)
newwarp = cv2.warpPerspective(color_warp, Minv, (image.shape[1], image.shape[0]))
# Combine the result with the original image
result = cv2.addWeighted(image, 1, newwarp, 0.3, 0)
return result
In [10]:
# Our function to process each image
def process_image(current_image):
if test_mode:
print_image(current_image, "Original Image")
binary_color_combined = extract_combined_binary(current_image)
if test_mode:
print_gray_image(binary_color_combined, "Thresholded Binary")
warped_binary, M, Minv = perform_binary_warp(binary_color_combined)
if test_mode:
print_gray_image(warped_binary, "Warped Binary with histogram and lane visualization")
left_fit, right_fit, leftx, lefty, rightx, righty = fit_polynomial_with_sliding_windows(warped_binary)
left_radius, right_radius, ploty, offset_from_center = measure_radius_of_curvature(warped_binary, left_fit, right_fit, leftx, lefty, rightx, righty)
result = project_detection_on_lane(warped_binary, current_image, left_fit, right_fit, ploty, Minv)
left_rad_round = "%.2f" % left_radius
right_rad_round = "%.2f" % right_radius
offset_from_center = "%.2f" % offset_from_center
cv2.putText(result,
"Radii of Curvature " + str((left_rad_round, right_rad_round)) + " m",
(300,670), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2 ,cv2.LINE_AA)
cv2.putText(result,
"Offset from img center " + str(offset_from_center) + " m",
(400,570), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2 ,cv2.LINE_AA)
return result
# Set to True for visualizing and False for generating video
test_mode = True
if test_mode:
test_images_path = '/Users/Akshay/projects/carnd/CarND-Advanced-Lane-Lines/test_images/test*.jpg'
test_images = glob.glob(test_images_path)
for img in test_images:
current_image = mpimg.imread(img)
print_image(process_image(current_image))
In [11]:
if not test_mode:
output = './p4.mp4'
clip1 = VideoFileClip("project_video.mp4")
white_clip = clip1.fl_image(process_image) # NOTE: this function expects color images!!
%time white_clip.write_videofile(output, audio=False)